{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3a2aff24",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4162997f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This script is using the following backgammon library to generate the samples\n",
    "# https://github.com/softwerks/backgammon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bfc22afb",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install backgammon\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "98132c9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import math\n",
    "import os\n",
    "import pathlib\n",
    "import random\n",
    "import time\n",
    "import yaml\n",
    "\n",
    "from copy import deepcopy\n",
    "from itertools import pairwise\n",
    "from typing import List, NamedTuple, Optional, Tuple\n",
    "\n",
    "import backgammon\n",
    "import numpy as np\n",
    "import openai\n",
    "import pandas as pd\n",
    "from backgammon.backgammon import STARTING_MATCH_ID, STARTING_POSITION_ID, Move, Player\n",
    "\n",
    "from sklearn.metrics import confusion_matrix\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8a9f6eb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_path = pathlib.Path(\"../evals/evals/registry/evals/\")\n",
    "eval_data_path = pathlib.Path(\"../evals/evals/registry/data/\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "68bc0d6e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('cAgAAAAAAAAA', '4HPwATDgc/ABMA')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "STARTING_MATCH_ID, STARTING_POSITION_ID\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "50262b0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "openai.api_key = os.environ.get(\"OPENAI_API_KEY\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cdde0fb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def roll_and_play(b) -> backgammon.Backgammon:\n",
    "    \"\"\"Generates \"naive\" backgammon moves for random dice rolls and plays them.\n",
    "\n",
    "    Args:\n",
    "        b (backgammon.Backgammon): a backgammon instance\n",
    "\n",
    "    Returns:\n",
    "        backgammon.Backgammon: board after moves habe been applied\n",
    "    \"\"\"\n",
    "\n",
    "    b.roll()\n",
    "    plays = [play for play in b.generate_plays()]\n",
    "\n",
    "    # cannot move\n",
    "    if not plays:\n",
    "        return b.end_turn()\n",
    "\n",
    "    # naive way of \"evaluating\" a position with as few slots as possible\n",
    "    # to have fewer legal moves availabe when creating the eval data\n",
    "    idx_to_slots = [\n",
    "        (idx, (np.array(p.position.board_points) == 1).sum())\n",
    "        for idx, p in enumerate(plays)\n",
    "    ]\n",
    "    idx_to_slots.sort(key=lambda x: x[1])\n",
    "    chosen_play = plays[idx_to_slots[0][0]]\n",
    "\n",
    "    moves = tuple(((x.source, x.destination) for x in chosen_play.moves))\n",
    "    b.play(moves)\n",
    "\n",
    "    return b\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7296162f",
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_SAMPLES = 10_000\n",
    "\n",
    "b = backgammon.Backgammon()\n",
    "\n",
    "eval_data = []\n",
    "for _ in range(MAX_SAMPLES):\n",
    "    b = backgammon.Backgammon()  # starting pos\n",
    "\n",
    "    rounds = random.choice([4, 6])\n",
    "    for x in range(rounds):\n",
    "        b = roll_and_play(b)\n",
    "\n",
    "    # roll once more to get the final board state\n",
    "    b.roll()\n",
    "\n",
    "    position_id, match_id = b.encode().split(\":\")\n",
    "\n",
    "    # moves which would hit the opponent (again)\n",
    "    hit_moves = [\n",
    "        # moves: namedtuples in df does not play nice when storing\n",
    "        [tuple(m) for m in play.moves]\n",
    "        for play in b.generate_plays()\n",
    "        if play.position.opponent_bar > b.position.opponent_bar\n",
    "    ]\n",
    "\n",
    "    eval_data.append(\n",
    "        {\n",
    "            \"match_id\": match_id,\n",
    "            \"position_id\": position_id,\n",
    "            \"encoded\": b.encode(),\n",
    "            \"dice\": b.match.dice,\n",
    "            \"rounds\": rounds,\n",
    "            \"hit_moves\": hit_moves,\n",
    "            \"can_hit\": any(hit_moves),\n",
    "            \"player_bar\": b.position.player_bar,\n",
    "            \"opponent_bar\": b.position.opponent_bar,\n",
    "        }\n",
    "    )\n",
    "\n",
    "df = pd.DataFrame(eval_data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9f7bebba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9888, 9)\n"
     ]
    }
   ],
   "source": [
    "df.drop_duplicates(subset=[\"encoded\"], inplace=True)\n",
    "print(df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "eebe1266",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_illegal_blocked_move_for_roll(b, roll) -> List[Tuple[Move, ...]]:\n",
    "    \"\"\"Returns a list of Move tuples given an individual dice roll - all those moves are not allowed\n",
    "    since they would move to a point with 2+ checkers of the opponent\n",
    "\n",
    "    Args:\n",
    "        b (backgammon.Backgammon): the board\n",
    "        roll (int): dice roll\n",
    "\n",
    "    Returns:\n",
    "        List[Tuple[Move, ...]]: List if Move tuples, e.g.\n",
    "            [Move(pips=5, source=23, destination=18), ... ]\n",
    "    \"\"\"\n",
    "\n",
    "    df = pd.DataFrame(b.position.board_points, columns=[\"board_points\"])\n",
    "    df[\"board\"] = list(range(0, 24))\n",
    "    # more intuitive\n",
    "    df = df.reindex(index=df.index[::-1])\n",
    "    df[\"shifted_board_points\"] = df[\"board_points\"].shift(roll)\n",
    "    df[\"illegal\"] = (\n",
    "        (df.board_points * df.shifted_board_points < 0)\n",
    "        & (df.shifted_board_points > 0)\n",
    "        & (df.board_points < -1)\n",
    "    )\n",
    "    df[\"source\"] = df.board + roll\n",
    "    df[\"destination\"] = df.board\n",
    "    df[\"pips\"] = roll\n",
    "\n",
    "    return list(\n",
    "        df[df.illegal == True][[\"pips\", \"source\", \"destination\"]].itertuples(\n",
    "            name=\"Move\", index=False\n",
    "        )\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "97286430",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_illegal_moves(row) -> Tuple[Move, ...]:\n",
    "    \"\"\"Constructd a \"valid\" illegal move.\n",
    "\n",
    "    Ignores moves that pass through, e.g. 7/6/5 (not make it too complicated)\n",
    "\n",
    "    Args:\n",
    "        row (row): Pandas Series\n",
    "\n",
    "    Returns:\n",
    "        Tuple[Move, ...]: The illegal Move\n",
    "    \"\"\"\n",
    "\n",
    "    b = backgammon.Backgammon(position_id=row.position_id, match_id=row.match_id)\n",
    "    all_plays = [play for play in b.generate_plays()]\n",
    "\n",
    "    # ignore moves where we move the same checker twice - regardless if the slot is empty or not\n",
    "    def checker_is_moving_through(moves):\n",
    "        return [p[0].destination == p[1].source for p in pairwise(moves)]\n",
    "\n",
    "    plays = [\n",
    "        play for play in all_plays if not any(checker_is_moving_through(play.moves))\n",
    "    ]\n",
    "\n",
    "    illegal_moves = {\n",
    "        roll: get_illegal_blocked_move_for_roll(b, roll) for roll in b.match.dice\n",
    "    }\n",
    "\n",
    "    # everygthing is valid\n",
    "    if not any(illegal_moves.values()) or not (plays):\n",
    "        return []\n",
    "\n",
    "    # choose randon play to manipulate\n",
    "    random_play = random.choice(plays)\n",
    "\n",
    "    new_illegal_move = list(random_play.moves)\n",
    "    for pos, move in enumerate(random_play.moves):\n",
    "        # do we have an invalid blocked move for this roll?\n",
    "        if not illegal_moves[move.pips]:\n",
    "            # not - we keep the original\n",
    "            continue\n",
    "        else:\n",
    "            # chose a random illegal move for the roll\n",
    "            new_illegal_move[pos] = random.choice(illegal_moves[move.pips])\n",
    "            break\n",
    "\n",
    "    return [tuple(x) for x in new_illegal_move]\n",
    "\n",
    "\n",
    "df[\"illegal_move\"] = df.apply(lambda row: get_illegal_moves(row), axis=1)\n",
    "df[\"has_illegal_move\"] = df.illegal_move.astype(bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0dcef42d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>match_id</th>\n",
       "      <th>position_id</th>\n",
       "      <th>encoded</th>\n",
       "      <th>dice</th>\n",
       "      <th>rounds</th>\n",
       "      <th>hit_moves</th>\n",
       "      <th>can_hit</th>\n",
       "      <th>player_bar</th>\n",
       "      <th>opponent_bar</th>\n",
       "      <th>illegal_move</th>\n",
       "      <th>has_illegal_move</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3641</th>\n",
       "      <td>cIgEAAAAAAAA</td>\n",
       "      <td>jJ/BATB4zeABMA</td>\n",
       "      <td>jJ/BATB4zeABMA:cIgEAAAAAAAA</td>\n",
       "      <td>(1, 1)</td>\n",
       "      <td>6</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[(1, 12, 11), (1, 3, 2), (1, 3, 2), (1, 7, 6)]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4563</th>\n",
       "      <td>cIgGAAAAAAAA</td>\n",
       "      <td>mOcPADB4PuABMA</td>\n",
       "      <td>mOcPADB4PuABMA:cIgGAAAAAAAA</td>\n",
       "      <td>(5, 1)</td>\n",
       "      <td>6</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[(5, 5, 0), (1, 23, 22)]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9221</th>\n",
       "      <td>cIgNAAAAAAAA</td>\n",
       "      <td>jJ+DATCwz+ABMA</td>\n",
       "      <td>jJ+DATCwz+ABMA:cIgNAAAAAAAA</td>\n",
       "      <td>(3, 3)</td>\n",
       "      <td>4</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8039</th>\n",
       "      <td>cIgSAAAAAAAA</td>\n",
       "      <td>HOfgATCY5x4AMA</td>\n",
       "      <td>HOfgATCY5x4AMA:cIgSAAAAAAAA</td>\n",
       "      <td>(5, 4)</td>\n",
       "      <td>6</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[(5, 23, 18), (4, 8, 4)]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7935</th>\n",
       "      <td>cAgNAAAAAAAA</td>\n",
       "      <td>sM/gATBkZ/ABMA</td>\n",
       "      <td>sM/gATBkZ/ABMA:cAgNAAAAAAAA</td>\n",
       "      <td>(2, 3)</td>\n",
       "      <td>4</td>\n",
       "      <td>[]</td>\n",
       "      <td>False</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>[(2, 2, 0), (3, 12, 9)]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          match_id     position_id                      encoded    dice  \\\n",
       "3641  cIgEAAAAAAAA  jJ/BATB4zeABMA  jJ/BATB4zeABMA:cIgEAAAAAAAA  (1, 1)   \n",
       "4563  cIgGAAAAAAAA  mOcPADB4PuABMA  mOcPADB4PuABMA:cIgGAAAAAAAA  (5, 1)   \n",
       "9221  cIgNAAAAAAAA  jJ+DATCwz+ABMA  jJ+DATCwz+ABMA:cIgNAAAAAAAA  (3, 3)   \n",
       "8039  cIgSAAAAAAAA  HOfgATCY5x4AMA  HOfgATCY5x4AMA:cIgSAAAAAAAA  (5, 4)   \n",
       "7935  cAgNAAAAAAAA  sM/gATBkZ/ABMA  sM/gATBkZ/ABMA:cAgNAAAAAAAA  (2, 3)   \n",
       "\n",
       "      rounds hit_moves  can_hit  player_bar  opponent_bar  \\\n",
       "3641       6        []    False           0             0   \n",
       "4563       6        []    False           0             0   \n",
       "9221       4        []    False           0             0   \n",
       "8039       6        []    False           0             0   \n",
       "7935       4        []    False           0             0   \n",
       "\n",
       "                                        illegal_move  has_illegal_move  \n",
       "3641  [(1, 12, 11), (1, 3, 2), (1, 3, 2), (1, 7, 6)]              True  \n",
       "4563                        [(5, 5, 0), (1, 23, 22)]              True  \n",
       "9221                                              []             False  \n",
       "8039                        [(5, 23, 18), (4, 8, 4)]              True  \n",
       "7935                         [(2, 2, 0), (3, 12, 9)]              True  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(n=5)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "cd40a1af",
   "metadata": {},
   "source": [
    "## Sample Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f288f5c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 300\n",
    "\n",
    "df[\"is_double\"] = df.dice.apply(lambda x: x[0] == x[1])\n",
    "can_hit_sample_df = df.groupby(\"can_hit\").sample(n=n, random_state=1)\n",
    "\n",
    "# avoid dupes and also coming from the bar to make it easier for the\n",
    "illegal_n = 500\n",
    "illegal_sample_df = (\n",
    "    df[(~df.index.isin(can_hit_sample_df.index)) & (df.player_bar == 0)]\n",
    "    .groupby(\"has_illegal_move\")\n",
    "    .sample(n=illegal_n, random_state=1)\n",
    ")\n",
    "\n",
    "can_hit_sample_df = can_hit_sample_df.sample(frac=1, random_state=1)\n",
    "illegal_sample_df = illegal_sample_df.sample(frac=1, random_state=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fee2cc3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pickles are available here: https://drive.google.com/drive/folders/1JqOt0yg0iSGjnWGuUXAuSi9X5XkZ6pTW?usp=sharing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e8cbfb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# store\n",
    "df.to_pickle(\"samples_df.p\")\n",
    "can_hit_sample_df.to_pickle(\"can_hit_sample_df.p\")\n",
    "illegal_sample_df.to_pickle(\"illegal_sample_df.p\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c318ef67",
   "metadata": {},
   "outputs": [],
   "source": [
    "# read back\n",
    "df = pd.read_pickle(\"samples_df.p\")\n",
    "can_hit_sample_df = pd.read_pickle(\"can_hit_sample_df.p\")\n",
    "illegal_sample_df = pd.read_pickle(\"illegal_sample_df.p\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "daac9ba0",
   "metadata": {},
   "source": [
    "### \"can we hit the opponent\"-sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0a8de33b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 Position ID: is/gATDgz4MBMA\n",
      "                 Match ID   : cAgVAAAAAAAA\n",
      " +13-14-15-16-17-18------19-20-21-22-23-24-+\n",
      " | X           O    |   | O        O  O  X |\n",
      " | X           O    |   | O              X |\n",
      " |                  |   | O                |\n",
      " |                  |   | O                |\n",
      " |                  |   | O                |\n",
      "v|                  |BAR|                  |\n",
      " |                  |   | 7                |\n",
      " | O           X    |   | X                |\n",
      " | O           X    |   | X                |\n",
      " | O           X    |   | X              O |\n",
      " | O           X    |   | X              O |\n",
      " +12-11-10--9--8--7-------6--5--4--3--2--1-+\n",
      "\n",
      "dice: (2, 5)\n",
      "\n",
      "these plays hit an opponents checker:\n",
      "24/22 13/8\n",
      "24/22 8/3\n"
     ]
    }
   ],
   "source": [
    "# sample a case\n",
    "# points are index based (!) -> +1 to match the board\n",
    "# cIgVAAAAAAAA\n",
    "\n",
    "sample = can_hit_sample_df[can_hit_sample_df.can_hit == True].sample(1, random_state=2)\n",
    "\n",
    "b = backgammon.Backgammon(\n",
    "    position_id=sample.position_id.item(), match_id=sample.match_id.item()\n",
    ")\n",
    "print(b)\n",
    "# which plays hit?\n",
    "print(f\"dice: {sample.dice.item()}\")\n",
    "print()\n",
    "\n",
    "print(\"these plays hit an opponents checker:\")\n",
    "for hit_moves in sample.hit_moves:\n",
    "    for rolls in hit_moves:\n",
    "        s = []\n",
    "        rolls.sort(key=lambda x: x[1], reverse=True)\n",
    "        for roll in rolls:\n",
    "            roll = Move(*roll)\n",
    "            s.append(f\"{roll.source +1}/{roll.destination+1}\")\n",
    "        print(\" \".join(s))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "ddf802c9",
   "metadata": {},
   "source": [
    "### \"is illegal play\"-sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ea922530",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 Position ID: zD3gATDg58HBAA\n",
      "                 Match ID   : cIgVAAAAAAAA\n",
      " +13-14-15-16-17-18------19-20-21-22-23-24-+\n",
      " | X              X |   | O  O     O       |\n",
      " | X              X |   | O  O     O       |\n",
      " | X                |   | O  O             |\n",
      " |                  |   | O                |\n",
      " |                  |   |                  |\n",
      "v|                  |BAR|                  |\n",
      " |                  |   | 6                |\n",
      " | O           X    |   | X                |\n",
      " | O           X    |   | X                |\n",
      " | O           X    |   | X              O |\n",
      " | O           X    |   | X              O |\n",
      " +12-11-10--9--8--7-------6--5--4--3--2--1-+\n",
      "\n",
      "dice: (3, 5)\n",
      "\n",
      "this ia an illegal play:\n",
      "6/3 6/1\n"
     ]
    }
   ],
   "source": [
    "sample = illegal_sample_df[illegal_sample_df.has_illegal_move == True].sample(\n",
    "    1, random_state=1\n",
    ")\n",
    "b = backgammon.Backgammon(\n",
    "    position_id=sample.position_id.item(), match_id=sample.match_id.item()\n",
    ")\n",
    "print(b)\n",
    "# which plays hit?\n",
    "print(f\"dice: {sample.dice.item()}\")\n",
    "print()\n",
    "print(\"this ia an illegal play:\")\n",
    "\n",
    "s = []\n",
    "for illegal_move in sample.illegal_move:\n",
    "    for roll in illegal_move:\n",
    "        roll = Move(*roll)\n",
    "        s.append(f\"{roll.source +1}/{roll.destination+1}\")\n",
    "    print(\" \".join(s))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "62e34c7a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 Position ID: 4NvBAQPg84UBMA\n",
      "                 Match ID   : cIgNAAAAAAAA\n",
      " +13-14-15-16-17-18------19-20-21-22-23-24-+\n",
      " | X           O  O |   | O              X |\n",
      " | X           O  O |   | O              X |\n",
      " |             O    |   | O                |\n",
      " |                  |   | O                |\n",
      " |                  |   | O                |\n",
      "v|                  |BAR|                  |\n",
      " |             X    |   | X                |\n",
      " |             X    |   | X                |\n",
      " | O           X    |   | X                |\n",
      " | O           X    |   | X  O             |\n",
      " | O        X  X    |   | X  O             |\n",
      " +12-11-10--9--8--7-------6--5--4--3--2--1-+\n",
      "\n",
      "dice: (3, 3)\n",
      "\n",
      "this is an illegal play:\n",
      "8/5 6/3 6/3 13/10\n"
     ]
    }
   ],
   "source": [
    "# another example - rolling a double:\n",
    "sample = illegal_sample_df[\n",
    "    (illegal_sample_df.has_illegal_move == True) & (illegal_sample_df.is_double == True)\n",
    "].sample(1, random_state=1)\n",
    "\n",
    "b = backgammon.Backgammon(\n",
    "    position_id=sample.position_id.item(), match_id=sample.match_id.item()\n",
    ")\n",
    "print(b)\n",
    "# which plays hit?\n",
    "print(f\"dice: {sample.dice.item()}\")\n",
    "print()\n",
    "print(\"this is an illegal play:\")\n",
    "\n",
    "s = []\n",
    "for illegal_move in sample.illegal_move:\n",
    "    for roll in illegal_move:\n",
    "        roll = Move(*roll)\n",
    "        s.append(f\"{roll.source +1}/{roll.destination+1}\")\n",
    "    print(\" \".join(s))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "3d61143c",
   "metadata": {},
   "source": [
    "## Generate promts / eval data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2c470984",
   "metadata": {},
   "outputs": [],
   "source": [
    "BOARD_POSITIONS = [x for x in reversed(range(1, 25))]\n",
    "\n",
    "GPT_PLAYER_NAME = \"backgammonGPT\"\n",
    "\n",
    "SYSTEM_CONTENT = (\n",
    "    f\"You are {GPT_PLAYER_NAME} - a backgammon engine. \"\n",
    "    \"Answer the following True/False question with the provided backgammon board state and the provided dice rolls. \"\n",
    "    f\"It is {GPT_PLAYER_NAME}'s turn.\"\n",
    ")\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "95fcbab6",
   "metadata": {},
   "source": [
    "### hit or not?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d0fa3a3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_hit_id = \"backgammon-can-hit\"\n",
    "\n",
    "json_data = []\n",
    "for pos, row in can_hit_sample_df.iterrows():\n",
    "    chat_gpt_player = []\n",
    "    opponent_player = []\n",
    "\n",
    "    b = backgammon.Backgammon(position_id=row.position_id, match_id=row.match_id)\n",
    "\n",
    "    for pos, board_points in enumerate(reversed(b.position.board_points)):\n",
    "        if not board_points:\n",
    "            continue\n",
    "\n",
    "        if board_points > 0:\n",
    "            chat_gpt_player.append(\n",
    "                f\"{board_points} checker{'s'[:board_points^1]} on the {BOARD_POSITIONS[pos]} point\"\n",
    "            )\n",
    "        else:\n",
    "            opponent_player.append(\n",
    "                f\"{abs(board_points)} checker{'s'[:abs(board_points)^1]} on the {BOARD_POSITIONS[pos]} point\"\n",
    "            )\n",
    "\n",
    "    chatgpt_setup = \", \".join(chat_gpt_player)\n",
    "    opponent_setup = \", \".join(opponent_player)\n",
    "\n",
    "    board_eval = f\"The backgammon board's position id is {row.position_id} and the match id is {row.match_id}\"\n",
    "    question = (\n",
    "        f\"{GPT_PLAYER_NAME} is rolling a {row.dice[0]} and a {row.dice[1]}. \"\n",
    "        f\"Can {GPT_PLAYER_NAME} hit one of the opponent's checkers?\"\n",
    "    )\n",
    "    true_false_only_reminder = \"Provide your reasoning step by step, and at the end, write your final answer, True or False, enclosed in square brackets.\"\n",
    "\n",
    "    user_content = (\n",
    "        f\"{board_eval}. {GPT_PLAYER_NAME} has {chatgpt_setup}. \"\n",
    "        f\"The opponent has {opponent_setup}. {question} {true_false_only_reminder}\"\n",
    "    )\n",
    "\n",
    "    messages = [\n",
    "        {\"role\": \"system\", \"content\": SYSTEM_CONTENT},\n",
    "        {\"role\": \"user\", \"content\": user_content},\n",
    "    ]\n",
    "\n",
    "    json_data.append({\"input\": messages, \"ideal\": f\"[{row.can_hit}]\"})\n",
    "\n",
    "# write can_hit samples\n",
    "backgammon_folder = eval_data_path / \"backgammon\"\n",
    "backgammon_folder.mkdir(parents=True, exist_ok=True)\n",
    "backgammon_can_hit = backgammon_folder / f\"{eval_hit_id}.jsonl\"\n",
    "\n",
    "with open(backgammon_can_hit, \"w\") as f:\n",
    "    for entry in json_data:\n",
    "        json.dump(entry, f)\n",
    "        f.write(\"\\n\")\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "29b88e31",
   "metadata": {},
   "source": [
    "### illegal move or not?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "2861eb70",
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_illegal_id = \"backgammon-illegal-move\"\n",
    "\n",
    "\n",
    "def get_json_data(df, is_few_shot=False):\n",
    "    json_data = []\n",
    "    for pos, row in df.iterrows():\n",
    "        chat_gpt_player = []\n",
    "        opponent_player = []\n",
    "        b = backgammon.Backgammon(position_id=row.position_id, match_id=row.match_id)\n",
    "\n",
    "        for pos, board_points in enumerate(reversed(b.position.board_points)):\n",
    "            if not board_points:\n",
    "                continue\n",
    "\n",
    "            if board_points > 0:\n",
    "                chat_gpt_player.append(\n",
    "                    f\"{board_points} checker{'s'[:board_points^1]} on the {BOARD_POSITIONS[pos]} point\"\n",
    "                )\n",
    "            else:\n",
    "                opponent_player.append(\n",
    "                    f\"{abs(board_points)} checker{'s'[:abs(board_points)^1]} on the {BOARD_POSITIONS[pos]} point\"\n",
    "                )\n",
    "\n",
    "        chatgpt_setup = \", \".join(chat_gpt_player)\n",
    "        opponent_setup = \", \".join(opponent_player)\n",
    "\n",
    "        board_eval = f\"The backgammon board's position id is {row.position_id} and the match id is {row.match_id}\"\n",
    "\n",
    "        moves_strings = []\n",
    "        if row.has_illegal_move:\n",
    "            moves = row.illegal_move\n",
    "            moves = [Move(*x) for x in moves]\n",
    "        else:\n",
    "            plays = [play for play in b.generate_plays()]\n",
    "            random_play = random.choice(plays)\n",
    "            moves = list(random_play.moves)\n",
    "\n",
    "        moves.sort(key=lambda x: x.source, reverse=True)\n",
    "        for roll in moves:\n",
    "            moves_strings.append(f\"{roll.source +1}/{roll.destination+1}\")\n",
    "\n",
    "        illegal_move_question = f\"Is {' '.join(moves_strings)} an illegal play?\"\n",
    "\n",
    "        question = f\"{GPT_PLAYER_NAME} is rolling a {row.dice[0]} and a {row.dice[1]}. {illegal_move_question}\"\n",
    "\n",
    "        true_false_only_reminder = \"Provide your reasoning step by step, and at the end, write your final answer, True or False, enclosed in square brackets.\"\n",
    "\n",
    "        user_content = (\n",
    "            f\"{board_eval}. {GPT_PLAYER_NAME} has {chatgpt_setup}. \"\n",
    "            f\"The opponent has {opponent_setup}. {question} {true_false_only_reminder}\"\n",
    "        )\n",
    "\n",
    "        if is_few_shot:\n",
    "            messages = [\n",
    "                {\"role\": \"system\", \"content\": user_content, \"name\": \"example_user\"},\n",
    "                {\n",
    "                    \"role\": \"system\",\n",
    "                    \"content\": f\"[{row.has_illegal_move}]\",\n",
    "                    \"name\": \"example_assistant\",\n",
    "                },\n",
    "            ]\n",
    "        else:\n",
    "            messages = [\n",
    "                {\"role\": \"system\", \"content\": SYSTEM_CONTENT},\n",
    "                {\"role\": \"user\", \"content\": user_content},\n",
    "            ]\n",
    "\n",
    "        # TODO: check how to phrase this\n",
    "        # negate has_illegal_move - so we can ask if it is a legal play\n",
    "        if is_few_shot:\n",
    "            json_data.append({\"sample\": messages})\n",
    "        else:\n",
    "            json_data.append({\"input\": messages, \"ideal\": f\"[{row.has_illegal_move}]\"})\n",
    "\n",
    "    return json_data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "11ad1ad3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# full set\n",
    "# few shot version omitted here\n",
    "json_data = get_json_data(illegal_sample_df)\n",
    "\n",
    "# write can_hit samples\n",
    "backgammon_folder = eval_data_path / \"backgammon\"\n",
    "backgammon_folder.mkdir(parents=True, exist_ok=True)\n",
    "backgammon_illegal_move = backgammon_folder / f\"{eval_illegal_id}.jsonl\"\n",
    "\n",
    "with open(backgammon_illegal_move, \"w\") as f:\n",
    "    for entry in json_data:\n",
    "        json.dump(entry, f)\n",
    "        f.write(\"\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9bfcf6ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "registry_yaml = {}\n",
    "\n",
    "# can hit registry\n",
    "registry_id = f\"{eval_hit_id}.dev.v0\"\n",
    "registry_yaml[eval_hit_id] = {\n",
    "    \"id\": registry_id,\n",
    "    \"metrics\": [\"accuracy\"],\n",
    "    \"description\": \"Evaluate if a checker can be hit given the a board state and a dice roll.\",\n",
    "}\n",
    "registry_yaml[registry_id] = {\n",
    "    \"class\": \"evals.elsuite.basic.includes:Includes\",\n",
    "    \"args\": {\"samples_jsonl\": f\"backgammon/{eval_hit_id}.jsonl\"},\n",
    "}\n",
    "\n",
    "# illegal move reg\n",
    "registry_illegal_id = f\"{eval_illegal_id}.dev.v0\"\n",
    "registry_yaml[eval_illegal_id] = {\n",
    "    \"id\": registry_illegal_id,\n",
    "    \"metrics\": [\"accuracy\"],\n",
    "    \"description\": \"Evaluate if the the given move is illegal given a board state, a dice roll and a suggested play.\",\n",
    "}\n",
    "registry_yaml[registry_illegal_id] = {\n",
    "    \"class\": \"evals.elsuite.basic.includes:Includes\",\n",
    "    \"args\": {\"samples_jsonl\": f\"backgammon/{eval_illegal_id}.jsonl\"},\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "5b7e1460",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(eval_path / \"backgammon.yaml\", \"w\") as f:\n",
    "    yaml.dump(registry_yaml, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0498f7ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "#%%capture\n",
    "!oaieval gpt-3.5-turbo backgammon-can-hit --record_path logs/bg_can_hit_4.log --no-cache --max_samples 30\n",
    "!oaieval gpt-3.5-turbo backgammon-illegal-move --record_path logs/bg_illegal_move_4.log --max_samples 30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f188f202",
   "metadata": {},
   "outputs": [],
   "source": [
    "#[2023-06-15 20:09:01,945] [registry.py:262] Loading registry from /Users/bakebrain/src/evals/evals/registry/evals\n",
    "#[2023-06-15 20:09:02,249] [registry.py:262] Loading registry from /Users/bakebrain/.evals/evals\n",
    "#[2023-06-15 20:09:02,251] [oaieval.py:138] Run started: 230615180902CTNID7G6\n",
    "#[2023-06-15 20:09:02,253] [data.py:83] Fetching backgammon/backgammon-can-hit.jsonl\n",
    "#[2023-06-15 20:09:02,255] [eval.py:33] Evaluating 30 samples\n",
    "#[2023-06-15 20:09:02,264] [eval.py:139] Running in threaded mode with 10 threads!\n",
    "# 70%|██████████████████████████████             | 21/30 [00:30<00:09,  1.04s/it][2023-06-15 20:09:32,697] [_common.py:105] Backing off openai_chat_completion_create_retrying(...) for 1.3s (openai.error.RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 408ec222c341c44a533c64c9e143028d in your message.))\n",
    "#100%|███████████████████████████████████████████| 30/30 [01:34<00:00,  3.15s/it]\n",
    "#[2023-06-15 20:10:36,801] [record.py:341] Final report: {'accuracy': 0.4666666666666667, 'boostrap_std': 0.092232532221554}. Logged to logs/bg_can_hit_4.log\n",
    "#[2023-06-15 20:10:36,801] [oaieval.py:177] Final report:\n",
    "#[2023-06-15 20:10:36,802] [oaieval.py:179] accuracy: 0.4666666666666667\n",
    "#[2023-06-15 20:10:36,802] [oaieval.py:179] boostrap_std: 0.092232532221554\n",
    "#[2023-06-15 20:10:36,806] [record.py:330] Logged 60 rows of events to logs/bg_can_hit_4.log: insert_time=4.121ms\n",
    "#[2023-06-15 20:10:37,756] [registry.py:262] Loading registry from /Users/bakebrain/src/evals/evals/registry/evals\n",
    "#[2023-06-15 20:10:38,086] [registry.py:262] Loading registry from /Users/bakebrain/.evals/evals\n",
    "#[2023-06-15 20:10:38,088] [oaieval.py:138] Run started: 230615181038KNSB73VV\n",
    "#[2023-06-15 20:10:38,089] [data.py:83] Fetching backgammon/backgammon-illegal-move.jsonl\n",
    "#[2023-06-15 20:10:38,093] [eval.py:33] Evaluating 30 samples\n",
    "#[2023-06-15 20:10:38,103] [eval.py:139] Running in threaded mode with 10 threads!\n",
    "#100%|███████████████████████████████████████████| 30/30 [00:39<00:00,  1.31s/it]\n",
    "#[2023-06-15 20:11:17,357] [record.py:341] Final report: {'accuracy': 0.6666666666666666, 'boostrap_std': 0.08635613598478237}. Logged to logs/bg_illegal_move_4.log\n",
    "#[2023-06-15 20:11:17,358] [oaieval.py:177] Final report:\n",
    "#[2023-06-15 20:11:17,358] [oaieval.py:179] accuracy: 0.6666666666666666\n",
    "#[2023-06-15 20:11:17,358] [oaieval.py:179] boostrap_std: 0.08635613598478237\n",
    "#[2023-06-15 20:11:17,362] [record.py:330] Logged 60 rows of events to logs/bg_illegal_move_4.log: insert_time=4.318ms\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "7ca38915",
   "metadata": {},
   "source": [
    "## eval\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "7a36041f",
   "metadata": {},
   "source": [
    "### can we hit?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "7ca96858",
   "metadata": {},
   "outputs": [],
   "source": [
    "events = \"logs/bg_can_hit_4.log\"\n",
    "\n",
    "with open(events, \"r\") as f:\n",
    "    events_df = pd.read_json(f, lines=True)\n",
    "\n",
    "matches_df = events_df[events_df.type == \"match\"].reset_index(drop=True)\n",
    "matches_df = matches_df.join(pd.json_normalize(matches_df.data))\n",
    "\n",
    "expected_strs = matches_df.expected.values\n",
    "expected = expected_strs == \"[True]\"\n",
    "correct = matches_df.correct.values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "8400ab46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4666666666666667"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "correct.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "81b7952a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# is this data in the df actually\n",
    "y = np.where(correct, expected, np.logical_not(expected))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "845eccac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(y == expected).mean() == correct.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "cc700ed3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>pred</th>\n",
       "      <th>False</th>\n",
       "      <th>True</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>actual</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>False</th>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>21</td>\n",
       "      <td>9</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "pred    False  True  All\n",
       "actual                  \n",
       "False      10     5   15\n",
       "True       11     4   15\n",
       "All        21     9   30"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(expected, y, rownames=[\"actual\"], colnames=[\"pred\"], margins=True)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "a2b37dae",
   "metadata": {},
   "source": [
    "### is it an illegal play?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "a4999131",
   "metadata": {},
   "outputs": [],
   "source": [
    "events = \"logs/bg_illegal_move_4.log\"\n",
    "\n",
    "with open(events, \"r\") as f:\n",
    "    events_df = pd.read_json(f, lines=True)\n",
    "\n",
    "matches_df = events_df[events_df.type == \"match\"].reset_index(drop=True)\n",
    "matches_df = matches_df.join(pd.json_normalize(matches_df.data))\n",
    "\n",
    "expected_strs = matches_df.expected.values\n",
    "expected = expected_strs == \"[True]\"\n",
    "correct = matches_df.correct.values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "f810488c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6666666666666666"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "correct.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "a06233ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "y = np.where(correct, expected, np.logical_not(expected))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "08b447c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>pred</th>\n",
       "      <th>False</th>\n",
       "      <th>True</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>actual</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>False</th>\n",
       "      <td>14</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>22</td>\n",
       "      <td>8</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "pred    False  True  All\n",
       "actual                  \n",
       "False      14     2   16\n",
       "True        8     6   14\n",
       "All        22     8   30"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(expected, y, rownames=[\"actual\"], colnames=[\"pred\"], margins=True)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
